package com.liusy.serachengine.parserFile;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;

import org.apache.log4j.Logger;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.TableColumn;
import org.htmlparser.tags.TableRow;
import org.htmlparser.tags.TableTag;
import org.htmlparser.util.NodeList;

public class ParserHtml {
private static Logger log = Logger.getLogger(ParserHtml.class);
	   public static StringBuffer readHtml(String urlString) {
	      StringBuffer content = new StringBuffer("");
	      try {
	      File file = new File(urlString);
	      if(!file.exists())return content;
	      FileInputStream fis = null;
	         fis = new FileInputStream(file);
	         BufferedReader reader = new BufferedReader(new InputStreamReader(fis, "GBK"));//������ַ����Ҫע�⣬Ҫ����htmlͷ�ļ���һ�£����������� 
	         String line = null;
	         while ((line = reader.readLine()) != null) {
	            content.append(line + " ");
	         }
	         reader.close();
	         fis.close();
			} catch (Exception e) {
				log.error("parser HTML error:"+urlString, e);
			}
		      
	      return removeHtmlTag(content.toString());
	   }
	   
	   public static StringBuffer removeHtmlTag(String content)
	   {
		      Parser parser = Parser.createParser(content.toString(),"GBK");
		      Parser parser1 = Parser.createParser(content.toString(),"GBK");
		      StringBuffer content1 = new StringBuffer("");
		      try {
		    	   NodeList nodesDIV = parser.extractAllNodesThatMatch(new TagNameFilter ("DIV"));
		    	      if(nodesDIV!=null) {
		    	    	  for (int i = 0; i < nodesDIV.size(); i++) {
		    	    	  Node textnodeDIV = (Node) nodesDIV.elementAt(i);
		    	    	  content1.append(new String(textnodeDIV.toPlainTextString().getBytes("GBK")));
		    	    	  }
		    	    	  }
		  			NodeFilter tableFilter = new NodeClassFilter(TableTag.class);
						NodeList nodeList = parser1.extractAllNodesThatMatch(tableFilter);
						for (int i = 0; i < nodeList.size(); i++) {
							if (nodeList.elementAt(i) instanceof TableTag) {
								TableTag tag = (TableTag) nodeList.elementAt(i);
								TableRow[] rows = tag.getRows();
								if(null!=rows)
								for (int j = 0; j < rows.length; j++) {
									TableRow tr = (TableRow) rows[j];
									TableColumn[] td = tr.getColumns();
										for (int k = 0; k < td.length; k++) {
											content1.append(new String(td[k].toPlainTextString().trim()));
										}
								}

							}
						}

			} catch (Exception e) {
				log.error("removeHtmlTag error:", e);
				// TODO: handle exception
			}
			if(content1.toString().equals(""))return new StringBuffer(content);
			return content1;
		   
	   }



}

